/*
 * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved.
 * Description: mc2 trace
 * Author: huawei
 * Create: 2024-05-24
 */

#include "debug/dfx/trace/mc2_trace_utils.h"

#include <cmath>
#include <stdio.h>
#include "log.h"
#include "mmpa_api.h"
#include "securec.h"
#include "utrace_api.h"
#include "common/aicpu_hccl_common.h"
#include "common/aicpu_hccl_def.h"
#include "common/sqe_context.h"
#include "utils/mc2_aicpu_utils.h"

namespace {
constexpr uint32_t TRACE_RING_BUFF_SIZE = 128 * 1024;
constexpr uint32_t MAX_SQE_SUBMIT_NUM = 256U;
constexpr char UTRACE_SO[] = "libutrace.so";

using UtraceCreateWithAttrFunc = TraHandle (*)(TracerType, const char *, const TraceAttr *);
using UtraceGetHandleFunc = TraHandle (*)(TracerType, const char *);
using UtraceSubmitFunc = TraStatus (*)(TraHandle handle, const void *, uint32_t);
using UtraceEventCreateFunc = TraEventHandle (*)(const char *);
using UtraceEventDestroyFunc = void (*)(TraEventHandle);
using UtraceHandleDestroyFunc = void (*)(TraHandle);
using UtraceEventBindTraceFunc = TraStatus (*)(TraEventHandle, TraHandle);
using UtraceEventReportFunc = TraStatus (*)(TraEventHandle);
using UtraceSetGlobalAttrFunc = TraStatus (*)(const TraceGlobalAttr *);
using UtraceStructEntryCreateFunc = TraceStructEntry *(*)(const char *);
using UtraceStructItemFieldSetFunc = void (*)(TraceStructEntry *, const char *, uint8_t, uint8_t, uint64_t);
using UtraceStructItemArraySetFunc = void (*)(TraceStructEntry *, const char *, uint8_t, uint8_t, uint64_t);
using UtraceStructSetAttrFunc = void (*)(TraceStructEntry *, uint8_t, TraceAttr *);
using UtraceStructEntryDestroyFunc = void (*)(TraceStructEntry *);

void *g_soHandle = nullptr;
UtraceCreateWithAttrFunc g_utraceCreateWithAttr = nullptr;
UtraceGetHandleFunc g_utraceGetHandle = nullptr;
UtraceSubmitFunc g_utraceSubmit = nullptr;
UtraceEventCreateFunc g_utraceEventCreate = nullptr;
UtraceEventDestroyFunc g_utraceEventDestroy = nullptr;
UtraceHandleDestroyFunc g_utraceHandleDestroy = nullptr;
UtraceEventBindTraceFunc g_utraceEventBindTrace = nullptr;
UtraceEventReportFunc g_utraceEventReport = nullptr;
UtraceSetGlobalAttrFunc g_utraceSetGlobalAttr = nullptr;
UtraceStructEntryCreateFunc g_utraceStructEntryCreate = nullptr;
UtraceStructItemFieldSetFunc g_utraceStructItemFieldSet = nullptr;
UtraceStructItemArraySetFunc g_utraceStructItemArraySet = nullptr;
UtraceStructSetAttrFunc g_utraceStructSetAttr = nullptr;
UtraceStructEntryDestroyFunc g_utraceStructEntryDestroy = nullptr;

TraEventHandle g_eventHandle = -1;
TraHandle g_traceStrHandle = -1;
TraHandle g_traceTaskAndTilingDataHandle = -1;
TraHandle g_traceAicpuComDataHandle = -1;
TraHandle g_traceMsgInfoHandle = -1;
TraHandle g_traceSqeBatchInfoHandle = -1;
}

// 当msgNum不为2的幂时，会自动向上取2的幂
uint16_t MC2TraceUtils::GetMsgNum(size_t msgSize)
{
    float logValue = log(TRACE_RING_BUFF_SIZE / (msgSize + 16)) / log(2); // 16 预留字节 2 幂次
    return static_cast<uint16_t>(pow(2, floor(logValue))); // 2 幂次
}

HcclResult MC2TraceUtils::InitTraceStrHandle()
{
    // trace api存在限制: msgNum * msgSize <= 128k
    uint16_t strMsgNum = GetMsgNum(sizeof(TraceStr));
    TraceAttr attr = { 0 };
    attr = { false, strMsgNum, sizeof(TraceStr), nullptr };
    TraceStructEntry *TraceStrSt = g_utraceStructEntryCreate("TraceStr");
    g_utraceStructItemArraySet(TraceStrSt, "transmit", TRACE_STRUCT_ARRAY_TYPE_CHAR, TRACE_STRUCT_SHOW_MODE_CHAR,
        sizeof(TraceStr::transmit));
    g_utraceStructSetAttr(TraceStrSt, 0, &attr);
    g_traceStrHandle = g_utraceCreateWithAttr(TRACER_TYPE_SCHEDULE, "TraceStr", &attr);
    if (g_traceStrHandle < 0) {
        HCCL_ERROR("Create g_traceStrHandle failed, ret:%d", g_traceStrHandle);
        g_utraceStructEntryDestroy(TraceStrSt);
        return HCCL_E_INTERNAL;
    }
    TraStatus ret = g_utraceEventBindTrace(g_eventHandle, g_traceStrHandle);
    if (ret != TRACE_SUCCESS) {
        HCCL_ERROR("Bind g_traceStrHandle failed, ret:%d", ret);
        return HCCL_E_INTERNAL;
    }
    return HCCL_SUCCESS;
}

HcclResult MC2TraceUtils::InitTaskAndTilingDataHandle()
{
    uint16_t taskAndTilingDataMsgNum = GetMsgNum(sizeof(KFCtaskAndTilingTraceData));
    TraceAttr attr = { 0 };
    attr = { false, taskAndTilingDataMsgNum, sizeof(KFCtaskAndTilingTraceData), nullptr };
    TraceStructEntry *KFCtaskAndTilingTraceDataSt = g_utraceStructEntryCreate("KFCtaskAndTilingTraceData");
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "inputA", TRACE_STRUCT_FIELD_TYPE_UINT64,
        TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "outputC", TRACE_STRUCT_FIELD_TYPE_UINT64,
        TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "commOut", TRACE_STRUCT_FIELD_TYPE_UINT64,
        TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "context", TRACE_STRUCT_FIELD_TYPE_UINT64,
        TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "workSpace", TRACE_STRUCT_FIELD_TYPE_UINT64,
        TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "tilingData", TRACE_STRUCT_FIELD_TYPE_UINT64,
        TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "sendOff", TRACE_STRUCT_FIELD_TYPE_UINT64,
        TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "recvOff", TRACE_STRUCT_FIELD_TYPE_UINT64,
        TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "tailSendOff", TRACE_STRUCT_FIELD_TYPE_UINT64,
        TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "tailRecvOff", TRACE_STRUCT_FIELD_TYPE_UINT64,
        TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "sendCnt", TRACE_STRUCT_FIELD_TYPE_UINT64,
        TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "recvCnt", TRACE_STRUCT_FIELD_TYPE_UINT64,
        TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "tailSendCnt", TRACE_STRUCT_FIELD_TYPE_UINT64,
        TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "tailRecvCnt", TRACE_STRUCT_FIELD_TYPE_UINT64,
        TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "totalCnt", TRACE_STRUCT_FIELD_TYPE_UINT64,
        TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "turnNum", TRACE_STRUCT_FIELD_TYPE_UINT32,
        TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "tailNum", TRACE_STRUCT_FIELD_TYPE_UINT32,
        TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "stride", TRACE_STRUCT_FIELD_TYPE_UINT32,
        TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "workspaceOff", TRACE_STRUCT_FIELD_TYPE_UINT32,
        TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "notifyOff", TRACE_STRUCT_FIELD_TYPE_UINT32,
        TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "notifyBeginCnt", TRACE_STRUCT_FIELD_TYPE_UINT16,
        TRACE_STRUCT_SHOW_MODE_DEC, 2); // 2 uint16
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "notifyEndCnt", TRACE_STRUCT_FIELD_TYPE_UINT16,
        TRACE_STRUCT_SHOW_MODE_DEC, 2); // 2 uint16
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "useBufferType", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "funID", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "dataType", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "groupNum", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "reuseMode", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "commType", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "reduceOp", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "commOrder", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "waitPolicy", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "rspPolicy", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "exitPolicy", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "commAlg", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "taskType", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "debugMode", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "stepSize", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "sendArgIndex", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "recvArgIndex", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "commOutArgIndex", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "hasCommOut", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "reverse", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(KFCtaskAndTilingTraceDataSt, "reserve2", TRACE_STRUCT_FIELD_TYPE_UINT32,
        TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
    g_utraceStructSetAttr(KFCtaskAndTilingTraceDataSt, 0, &attr);
    g_traceTaskAndTilingDataHandle = g_utraceCreateWithAttr(TRACER_TYPE_SCHEDULE, "KFCtaskAndTilingTraceData", &attr);
    if (g_traceTaskAndTilingDataHandle < 0) {
        HCCL_ERROR("Create g_traceTaskAndTilingDataHandle failed, ret:%d", g_traceTaskAndTilingDataHandle);
        g_utraceStructEntryDestroy(KFCtaskAndTilingTraceDataSt);
        return HCCL_E_INTERNAL;
    }
    TraStatus ret = g_utraceEventBindTrace(g_eventHandle, g_traceTaskAndTilingDataHandle);
    if (ret != TRACE_SUCCESS) {
        HCCL_ERROR("Bind g_traceTaskAndTilingDataHandle failed, ret:%d", ret);
        return HCCL_E_INTERNAL;
    }
    return HCCL_SUCCESS;
}

HcclResult MC2TraceUtils::InitAicpuComDataHandle()
{
    uint16_t aicpuComMsgNum = GetMsgNum(sizeof(AicpuComTraceData));
    TraceAttr attr = { 0 };
    attr = { false, aicpuComMsgNum, sizeof(AicpuComTraceData), nullptr };
    TraceStructEntry *AicpuComTraceSt = g_utraceStructEntryCreate("AicpuComTraceData");
    g_utraceStructItemFieldSet(AicpuComTraceSt, "devId", TRACE_STRUCT_FIELD_TYPE_UINT32,
        TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
    g_utraceStructItemFieldSet(AicpuComTraceSt, "ssid", TRACE_STRUCT_FIELD_TYPE_UINT32,
        TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
    g_utraceStructItemFieldSet(AicpuComTraceSt, "rankId", TRACE_STRUCT_FIELD_TYPE_UINT32,
        TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
    g_utraceStructItemFieldSet(AicpuComTraceSt, "rankNum", TRACE_STRUCT_FIELD_TYPE_UINT32,
        TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
    g_utraceStructItemFieldSet(AicpuComTraceSt, "windowSize", TRACE_STRUCT_FIELD_TYPE_UINT64,
        TRACE_STRUCT_SHOW_MODE_DEC, 8); // 8 uint64
    g_utraceStructItemFieldSet(AicpuComTraceSt, "workSpaceAddr", TRACE_STRUCT_FIELD_TYPE_UINT64,
        TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
    g_utraceStructItemFieldSet(AicpuComTraceSt, "kfcNotifyId", TRACE_STRUCT_FIELD_TYPE_UINT64,
        TRACE_STRUCT_SHOW_MODE_DEC, 8); // 8 uint64
    g_utraceStructItemArraySet(AicpuComTraceSt, "eventIds", TRACE_STRUCT_ARRAY_TYPE_UINT32,
        TRACE_STRUCT_SHOW_MODE_DEC, 128); // 4 uint32 128 AC_MAX_RANK_NUM*uint32
    g_utraceStructItemArraySet(AicpuComTraceSt, "windowIn", TRACE_STRUCT_ARRAY_TYPE_UINT64,
        TRACE_STRUCT_SHOW_MODE_HEX, 256); // 8 uint64 256 AC_MAX_RANK_NUM*uint64
    g_utraceStructItemArraySet(AicpuComTraceSt, "windowOut", TRACE_STRUCT_ARRAY_TYPE_UINT64,
        TRACE_STRUCT_SHOW_MODE_HEX, 256); // 8 uint64 256 AC_MAX_RANK_NUM*uint64
    g_utraceStructItemArraySet(AicpuComTraceSt, "actualStreamId", TRACE_STRUCT_ARRAY_TYPE_INT32,
        TRACE_STRUCT_SHOW_MODE_DEC, 128); // 4 uint32 128 AC_MAX_RANK_NUM*uint32
    g_utraceStructItemArraySet(AicpuComTraceSt, "sqId", TRACE_STRUCT_ARRAY_TYPE_INT32,
        TRACE_STRUCT_SHOW_MODE_DEC, 128); // 4 uint32 128 AC_MAX_RANK_NUM*uint32
    g_utraceStructItemArraySet(AicpuComTraceSt, "aicpuOpNotifyAddress", TRACE_STRUCT_ARRAY_TYPE_UINT64,
        TRACE_STRUCT_SHOW_MODE_HEX, 16); // 8 uint64 16 2*uint64
    g_utraceStructItemArraySet(AicpuComTraceSt, "aicpuOpNotifyActualNotifyId", TRACE_STRUCT_ARRAY_TYPE_INT32,
        TRACE_STRUCT_SHOW_MODE_DEC, 8); // 4 uint32 8 2*uint32
    g_utraceStructItemFieldSet(AicpuComTraceSt, "clusterId", TRACE_STRUCT_FIELD_TYPE_INT32,
        TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
    g_utraceStructSetAttr(AicpuComTraceSt, 0, &attr);
    g_traceAicpuComDataHandle = g_utraceCreateWithAttr(TRACER_TYPE_SCHEDULE, "AicpuComTraceData", &attr);
    if (g_traceAicpuComDataHandle < 0) {
        HCCL_ERROR("Create g_traceAicpuComDataHandle failed, ret:%d", g_traceAicpuComDataHandle);
        g_utraceStructEntryDestroy(AicpuComTraceSt);
        return HCCL_E_INTERNAL;
    }
    TraStatus ret = g_utraceEventBindTrace(g_eventHandle, g_traceAicpuComDataHandle);
    if (ret != TRACE_SUCCESS) {
        HCCL_ERROR("Bind g_traceAicpuComDataHandle failed, ret:%d", ret);
        return HCCL_E_INTERNAL;
    }
    return HCCL_SUCCESS;
}

HcclResult MC2TraceUtils::InitMsgInfoHandle()
{
    uint16_t msgInfoNum = GetMsgNum(sizeof(AivAicpuOpParam));
    TraceAttr attr = { 0 };
    attr = { false, msgInfoNum, sizeof(AivAicpuOpParam), nullptr };
    TraceStructEntry *MsgInfoSt = g_utraceStructEntryCreate("AivAicpuOpParam");
    g_utraceStructItemFieldSet(MsgInfoSt, "commType", TRACE_STRUCT_FIELD_TYPE_UINT32,
        TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
    g_utraceStructItemFieldSet(MsgInfoSt, "opType", TRACE_STRUCT_FIELD_TYPE_UINT32,
        TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
    g_utraceStructItemFieldSet(MsgInfoSt, "sendBuffer", TRACE_STRUCT_FIELD_TYPE_UINT64,
        TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
    g_utraceStructItemFieldSet(MsgInfoSt, "recvBuffer", TRACE_STRUCT_FIELD_TYPE_UINT64,
        TRACE_STRUCT_SHOW_MODE_HEX, 8); // 8 uint64
    g_utraceStructItemFieldSet(MsgInfoSt, "count", TRACE_STRUCT_FIELD_TYPE_UINT64,
        TRACE_STRUCT_SHOW_MODE_DEC, 8); // 8 uint64
    g_utraceStructItemFieldSet(MsgInfoSt, "strideLen", TRACE_STRUCT_FIELD_TYPE_UINT64,
        TRACE_STRUCT_SHOW_MODE_DEC, 8); // 8 uint64
    g_utraceStructItemFieldSet(MsgInfoSt, "hcclDataType", TRACE_STRUCT_FIELD_TYPE_UINT32,
        TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
    g_utraceStructItemFieldSet(MsgInfoSt, "valid", TRACE_STRUCT_FIELD_TYPE_UINT32,
        TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
    g_utraceStructItemFieldSet(MsgInfoSt, "isLast", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(MsgInfoSt, "funID", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(MsgInfoSt, "sendCnt", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(MsgInfoSt, "rcvCnt", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(MsgInfoSt, "everyTurnRsp", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(MsgInfoSt, "everyTurnWait", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(MsgInfoSt, "totalTurnCnt", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(MsgInfoSt, "useBufferType", TRACE_STRUCT_FIELD_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
    g_utraceStructItemFieldSet(MsgInfoSt, "winOffset", TRACE_STRUCT_ARRAY_TYPE_UINT64,
        TRACE_STRUCT_SHOW_MODE_DEC, 8); // 8 uint64
    g_utraceStructItemArraySet(MsgInfoSt, "res", TRACE_STRUCT_ARRAY_TYPE_UINT8,
        TRACE_STRUCT_SHOW_MODE_DEC, 2); // 1 uint8 2 2*uint8
    g_utraceStructSetAttr(MsgInfoSt, 0, &attr);
    g_traceMsgInfoHandle = g_utraceCreateWithAttr(TRACER_TYPE_SCHEDULE, "AivAicpuOpParam", &attr);
    if (g_traceMsgInfoHandle < 0) {
        HCCL_ERROR("Create g_traceMsgInfoHandle failed, ret:%d", g_traceMsgInfoHandle);
        g_utraceStructEntryDestroy(MsgInfoSt);
        return HCCL_E_INTERNAL;
    }
    TraStatus ret = g_utraceEventBindTrace(g_eventHandle, g_traceMsgInfoHandle);
    if (ret != TRACE_SUCCESS) {
        HCCL_ERROR("Bind g_traceMsgInfoHandle failed, ret:%d", ret);
        return HCCL_E_INTERNAL;
    }
    return HCCL_SUCCESS;
}

HcclResult MC2TraceUtils::InitSqeBatchInfoHandle()
{
    uint16_t sqeBatchInfoMsgNum = GetMsgNum(sizeof(SqeBatchInfo));
    TraceAttr attr = { 0 };
    attr = { false, sqeBatchInfoMsgNum, sizeof(SqeBatchInfo), nullptr };
    TraceStructEntry *SqeBatchInfoSt = g_utraceStructEntryCreate("SqeBatchInfo");
    for (uint32_t i = 0; i < MAX_SQE_BATCH_SIZE; i++) {
        g_utraceStructItemFieldSet(SqeBatchInfoSt, "addr1High", TRACE_STRUCT_FIELD_TYPE_UINT32,
            TRACE_STRUCT_SHOW_MODE_HEX, 4); // 4 uint32
        g_utraceStructItemFieldSet(SqeBatchInfoSt, "addr1Low", TRACE_STRUCT_FIELD_TYPE_UINT32,
            TRACE_STRUCT_SHOW_MODE_HEX, 4); // 4 uint32
        g_utraceStructItemFieldSet(SqeBatchInfoSt, "addr2High", TRACE_STRUCT_FIELD_TYPE_UINT32,
            TRACE_STRUCT_SHOW_MODE_HEX, 4); // 4 uint32
        g_utraceStructItemFieldSet(SqeBatchInfoSt, "addr2Low", TRACE_STRUCT_FIELD_TYPE_UINT32,
            TRACE_STRUCT_SHOW_MODE_HEX, 4); // 4 uint32
        g_utraceStructItemFieldSet(SqeBatchInfoSt, "sqeHeadIdx", TRACE_STRUCT_FIELD_TYPE_UINT32,
            TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
        g_utraceStructItemFieldSet(SqeBatchInfoSt, "notifyId", TRACE_STRUCT_FIELD_TYPE_UINT32,
            TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
        g_utraceStructItemFieldSet(SqeBatchInfoSt, "length", TRACE_STRUCT_FIELD_TYPE_UINT32,
            TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
        g_utraceStructItemFieldSet(SqeBatchInfoSt, "partId", TRACE_STRUCT_FIELD_TYPE_UINT32,
            TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
        g_utraceStructItemFieldSet(SqeBatchInfoSt, "remoteRank", TRACE_STRUCT_FIELD_TYPE_UINT32,
            TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
        g_utraceStructItemFieldSet(SqeBatchInfoSt, "dataType", TRACE_STRUCT_FIELD_TYPE_UINT32,
            TRACE_STRUCT_SHOW_MODE_DEC, 4); // 4 uint32
        g_utraceStructItemFieldSet(SqeBatchInfoSt, "streamId", TRACE_STRUCT_FIELD_TYPE_UINT16,
            TRACE_STRUCT_SHOW_MODE_DEC, 2); // 2 uint16
        g_utraceStructItemFieldSet(SqeBatchInfoSt, "eventId", TRACE_STRUCT_FIELD_TYPE_UINT16,
            TRACE_STRUCT_SHOW_MODE_DEC, 2); // 2 uint16
        g_utraceStructItemFieldSet(SqeBatchInfoSt, "taskId", TRACE_STRUCT_FIELD_TYPE_UINT16,
            TRACE_STRUCT_SHOW_MODE_DEC, 2); // 2 uint16
        g_utraceStructItemFieldSet(SqeBatchInfoSt, "condValue", TRACE_STRUCT_FIELD_TYPE_UINT16,
            TRACE_STRUCT_SHOW_MODE_DEC, 2); // 2 uint16
        g_utraceStructItemFieldSet(SqeBatchInfoSt, "isLast", TRACE_STRUCT_FIELD_TYPE_UINT8,
            TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
        g_utraceStructItemFieldSet(SqeBatchInfoSt, "opCode", TRACE_STRUCT_FIELD_TYPE_UINT8,
            TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
        g_utraceStructItemFieldSet(SqeBatchInfoSt, "sqeNum", TRACE_STRUCT_FIELD_TYPE_UINT8,
            TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
        g_utraceStructItemFieldSet(SqeBatchInfoSt, "type", TRACE_STRUCT_FIELD_TYPE_UINT8,
            TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
        g_utraceStructItemFieldSet(SqeBatchInfoSt, "subType", TRACE_STRUCT_FIELD_TYPE_UINT8,
            TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
        g_utraceStructItemFieldSet(SqeBatchInfoSt, "valid", TRACE_STRUCT_FIELD_TYPE_UINT8,
            TRACE_STRUCT_SHOW_MODE_DEC, 1); // 1 uint8
        g_utraceStructItemArraySet(SqeBatchInfoSt, "reverse", TRACE_STRUCT_ARRAY_TYPE_UINT8,
            TRACE_STRUCT_SHOW_MODE_DEC, 10); // 1 uint8 10 10*uint8
    }
    g_utraceStructSetAttr(SqeBatchInfoSt, 0, &attr);
    g_traceSqeBatchInfoHandle = g_utraceCreateWithAttr(TRACER_TYPE_SCHEDULE, "SqeBatchInfo", &attr);
    if (g_traceSqeBatchInfoHandle < 0) {
        HCCL_ERROR("Create g_traceSqeBatchInfoHandle failed, ret:%d", g_traceSqeBatchInfoHandle);
        g_utraceStructEntryDestroy(SqeBatchInfoSt);
        return HCCL_E_INTERNAL;
    }
    TraStatus ret = g_utraceEventBindTrace(g_eventHandle, g_traceSqeBatchInfoHandle);
    if (ret != TRACE_SUCCESS) {
        HCCL_ERROR("Bind g_traceSqeBatchInfoHandle failed, ret:%d", ret);
        return HCCL_E_INTERNAL;
    }
    return HCCL_SUCCESS;
}

HcclResult MC2TraceUtils::InitTraceHandle()
{
    g_eventHandle = g_utraceEventCreate("mc2_dfx");
    if (g_eventHandle < 0) {
        HCCL_ERROR("Create mc2_dfx event handle failed");
        return HCCL_E_INTERNAL;
    }
    CHK_RET(InitTaskAndTilingDataHandle());
    CHK_RET(InitAicpuComDataHandle());
    CHK_RET(InitTraceStrHandle());
    CHK_RET(InitMsgInfoHandle());
    CHK_RET(InitSqeBatchInfoHandle());
    return HCCL_SUCCESS;
}

HcclResult MC2TraceUtils::InitFuncHandle()
{
    g_utraceCreateWithAttr = reinterpret_cast<UtraceCreateWithAttrFunc>(mmDlsym(g_soHandle, "UtraceCreateWithAttr"));
    CHK_PRT_RET(g_utraceCreateWithAttr == nullptr, HCCL_ERROR("Get g_utraceCreateWithAttr error: %s", mmDlerror()),
        HCCL_E_SYSCALL);
    g_utraceGetHandle = reinterpret_cast<UtraceGetHandleFunc>(mmDlsym(g_soHandle, "UtraceGetHandle"));
    CHK_PRT_RET(g_utraceGetHandle == nullptr, HCCL_ERROR("Get g_utraceGetHandle error: %s", mmDlerror()),
        HCCL_E_SYSCALL);
    g_utraceSubmit = reinterpret_cast<UtraceSubmitFunc>(mmDlsym(g_soHandle, "UtraceSubmit"));
    CHK_PRT_RET(g_utraceSubmit == nullptr, HCCL_ERROR("Get g_utraceSubmit error: %s", mmDlerror()), HCCL_E_SYSCALL);
    g_utraceEventCreate = reinterpret_cast<UtraceEventCreateFunc>(mmDlsym(g_soHandle, "UtraceEventCreate"));
    CHK_PRT_RET(g_utraceEventCreate == nullptr, HCCL_ERROR("Get g_utraceEventCreate error: %s", mmDlerror()),
        HCCL_E_SYSCALL);
    g_utraceEventDestroy = reinterpret_cast<UtraceEventDestroyFunc>(mmDlsym(g_soHandle, "UtraceEventDestroy"));
    CHK_PRT_RET(g_utraceEventDestroy == nullptr, HCCL_ERROR("Get g_utraceEventDestroy error: %s", mmDlerror()),
        HCCL_E_SYSCALL);
    g_utraceHandleDestroy = reinterpret_cast<UtraceHandleDestroyFunc>(mmDlsym(g_soHandle, "UtraceDestroy"));
    CHK_PRT_RET(g_utraceHandleDestroy == nullptr, HCCL_ERROR("Get g_utraceHandleDestroy error: %s", mmDlerror()),
        HCCL_E_SYSCALL);
    g_utraceEventBindTrace = reinterpret_cast<UtraceEventBindTraceFunc>(mmDlsym(g_soHandle, "UtraceEventBindTrace"));
    CHK_PRT_RET(g_utraceEventBindTrace == nullptr, HCCL_ERROR("Get g_utraceEventBindTrace error: %s", mmDlerror()),
        HCCL_E_SYSCALL);
    g_utraceEventReport = reinterpret_cast<UtraceEventReportFunc>(mmDlsym(g_soHandle, "UtraceEventReport"));
    CHK_PRT_RET(g_utraceEventReport == nullptr, HCCL_ERROR("Get g_utraceEventReport error: %s", mmDlerror()),
        HCCL_E_SYSCALL);
    g_utraceSetGlobalAttr = reinterpret_cast<UtraceSetGlobalAttrFunc>(mmDlsym(g_soHandle, "UtraceSetGlobalAttr"));
    CHK_PRT_RET(g_utraceSetGlobalAttr == nullptr, HCCL_ERROR("Get g_utraceSetGlobalAttr error: %s", mmDlerror()),
        HCCL_E_SYSCALL);
    g_utraceStructEntryCreate =
        reinterpret_cast<UtraceStructEntryCreateFunc>(mmDlsym(g_soHandle, "UtraceStructEntryCreate"));
    CHK_PRT_RET(g_utraceStructEntryCreate == nullptr,
        HCCL_ERROR("Get g_utraceStructEntryCreate error: %s", mmDlerror()), HCCL_E_SYSCALL);
    g_utraceStructItemFieldSet =
        reinterpret_cast<UtraceStructItemFieldSetFunc>(mmDlsym(g_soHandle, "UtraceStructItemFieldSet"));
    CHK_PRT_RET(g_utraceStructItemFieldSet == nullptr,
        HCCL_ERROR("Get g_utraceStructItemFieldSet error: %s", mmDlerror()), HCCL_E_SYSCALL);
    g_utraceStructItemArraySet =
        reinterpret_cast<UtraceStructItemArraySetFunc>(mmDlsym(g_soHandle, "UtraceStructItemArraySet"));
    CHK_PRT_RET(g_utraceStructItemArraySet == nullptr,
        HCCL_ERROR("Get g_utraceStructItemArraySet error: %s", mmDlerror()), HCCL_E_SYSCALL);
    g_utraceStructSetAttr = reinterpret_cast<UtraceStructSetAttrFunc>(mmDlsym(g_soHandle, "UtraceStructSetAttr"));
    CHK_PRT_RET(g_utraceStructSetAttr == nullptr, HCCL_ERROR("Get g_utraceStructSetAttr error: %s", mmDlerror()),
        HCCL_E_SYSCALL);
    g_utraceStructEntryDestroy =
        reinterpret_cast<UtraceStructEntryDestroyFunc>(mmDlsym(g_soHandle, "UtraceStructEntryDestroy"));
    CHK_PRT_RET(g_utraceStructEntryDestroy == nullptr,
        HCCL_ERROR("Get g_utraceStructEntryDestroy error: %s", mmDlerror()), HCCL_E_SYSCALL);
    return HCCL_SUCCESS;
}

__attribute__ ((constructor)) void DlopenTraceSo()
{
    if (g_soHandle == nullptr) {
        g_soHandle = mmDlopen(UTRACE_SO, RTLD_NOW | RTLD_GLOBAL);
        HCCL_INFO("mmDlopen success");
    }
}

HcclResult MC2TraceUtils::Init()
{
    if (g_soHandle == nullptr) {
        HCCL_WARNING("Cannot find so file: %s", UTRACE_SO);
        return HCCL_SUCCESS;
    }
    CHK_RET(InitFuncHandle());
    CHK_RET(InitTraceHandle());
    unsigned int hostpid = 0;
    unsigned int cpType = DEVDRV_PROCESS_CPTYPE_MAX;
    if (drvQueryProcessHostPid(getpid(), nullptr, nullptr, &hostpid, &cpType) != DRV_ERROR_NONE) {
        HCCL_ERROR("Get hostpid failed");
        return HCCL_E_DRV;
    }
    TraceGlobalAttr traceGlobalAttr = { 0 };
    traceGlobalAttr.saveMode = 1;
    traceGlobalAttr.deviceId = AicpuGetComContext()->devId;
    traceGlobalAttr.pid = hostpid;
    TraStatus ret = g_utraceSetGlobalAttr(&traceGlobalAttr);
    if (ret != TRACE_SUCCESS) {
        HCCL_ERROR("UtraceSetGlobalAttrFunc failed, ret:%d", ret);
        return HCCL_E_INTERNAL;
    }
    return HCCL_SUCCESS;
}

template <typename T> HcclResult MC2TraceUtils::Submit(const T * const traceData)
{
    if (g_soHandle == nullptr || g_utraceGetHandle == nullptr || g_utraceSubmit == nullptr) {
        return HCCL_SUCCESS;
    }
    CHK_PTR_NULL(traceData);
    std::string typeName;
    if (std::is_same<T, KFCtaskAndTilingTraceData>::value) {
        typeName = "KFCtaskAndTilingTraceData";
    } else if (std::is_same<T, AicpuComTraceData>::value) {
        typeName = "AicpuComTraceData";
    } else if (std::is_same<T, AivAicpuOpParam>::value) {
        typeName = "AivAicpuOpParam";
    } else if (std::is_same<T, SqeBatchInfo>::value) {
        typeName = "SqeBatchInfo";
    } else {
        HCCL_ERROR("find typename: %s failed", typeid(T).name());
        return HCCL_E_INTERNAL;
    }

    auto traceHandle = g_utraceGetHandle(TRACER_TYPE_SCHEDULE, typeName.c_str());
    if (traceHandle < 0) {
        HCCL_ERROR("getHandle %s failed, ret:%d", typeName.c_str(), traceHandle);
        return HCCL_E_INTERNAL;
    }
    auto traceRet = g_utraceSubmit(traceHandle, reinterpret_cast<const void *>(traceData), sizeof(T));
    if (traceRet != TRACE_SUCCESS) {
        HCCL_ERROR("submit %s failed, ret:%d", typeName.c_str(), traceRet);
        return HCCL_E_INTERNAL;
    }
    return HCCL_SUCCESS;
}

HcclResult MC2TraceUtils::Submit(AicpuComContext *ctx)
{
    if (g_soHandle == nullptr) {
        return HCCL_SUCCESS;
    }
    CHK_PTR_NULL(ctx);
    uint64_t startUsec = GetCurCpuTimestamp();
    AicpuComTraceData comTraceData;
    comTraceData.devId = ctx->devId;
    comTraceData.ssid = ctx->ssid;
    comTraceData.rankId = ctx->rankId;
    comTraceData.rankNum = ctx->rankNum;
    comTraceData.windowSize = ctx->windowSize;
    comTraceData.workSpaceAddr = ctx->workSpaceAddr;
    comTraceData.kfcNotifyId = ctx->kfcNotifyId;
    for (uint32_t i = 0; i < AC_MAX_RANK_NUM; i++) {
        comTraceData.eventIds[i] = ctx->eventIds[i];
        comTraceData.windowIn[i] = ctx->rankInfo[i].window;
        comTraceData.windowOut[i] = ctx->rankInfo[i].windowOut;
        comTraceData.actualStreamId[i] = ctx->streamInfo[i].actualStreamId;
        comTraceData.sqId[i] = ctx->streamInfo[i].sqId;
    }
    for (uint32_t i = 0; i < 2; i++) { // 2 aicpuOpNotify size
        comTraceData.aicpuOpNotifyAddress[i] = ctx->aicpuOpNotify[i].address;
        comTraceData.aicpuOpNotifyActualNotifyId[i] = ctx->aicpuOpNotify[i].actualNotifyId;
    }
    comTraceData.clusterId = ctx->clusterId;
    CHK_RET(Submit<AicpuComTraceData>(&comTraceData));
    if (MC2AicpuUtils::IsDebugModeEquals(*ctx, MC2_DEBUG_TIME_TAKEN)) {
        ctx->acprof[g_proxLoopCnt].traceCtxTime += GetCurCpuTimestamp() - startUsec;
    }
    return HCCL_SUCCESS;
}

HcclResult MC2TraceUtils::Submit(const KFCTask * const task, const HcclKFCTilingData * const tilingData)
{
    if (g_soHandle == nullptr) {
        return HCCL_SUCCESS;
    }
    CHK_PTR_NULL(task);
    CHK_PTR_NULL(tilingData);
    uint64_t startUsec = GetCurCpuTimestamp();
    KFCtaskAndTilingTraceData taskAndTilingData;
    error_t ret = memcpy_s(&taskAndTilingData, sizeof(KFCtaskAndTilingTraceData), task, sizeof(KFCTask));
    if (ret != EOK) {
        HCCL_ERROR("task memcpy_s fail ret:%d", ret);
        return HCCL_E_MEMORY;
    }
    ret = memcpy_s(reinterpret_cast<uint8_t *>(&taskAndTilingData) + sizeof(KFCTask),
        sizeof(KFCtaskAndTilingTraceData) - sizeof(KFCTask), tilingData, sizeof(HcclKFCTilingData));
    if (ret != EOK) {
        HCCL_ERROR("tilingData memcpy_s fail ret:%d", ret);
        return HCCL_E_MEMORY;
    }
    CHK_RET(Submit<KFCtaskAndTilingTraceData>(&taskAndTilingData));
    auto ctx = AicpuGetComContext();
    if (MC2AicpuUtils::IsDebugModeEquals(*ctx, MC2_DEBUG_TIME_TAKEN)) {
        ctx->acprof[g_proxLoopCnt].traceSubmitTime += GetCurCpuTimestamp() - startUsec;
    }
    return HCCL_SUCCESS;
}

HcclResult MC2TraceUtils::Submit(const std::string &traceStr)
{
    if (g_soHandle == nullptr) {
        return HCCL_SUCCESS;
    }
    return Submit(traceStr.c_str());
}

HcclResult MC2TraceUtils::Submit(const char *traceStr)
{
    if (g_soHandle == nullptr || g_utraceSubmit == nullptr) {
        return HCCL_SUCCESS;
    }
    uint64_t startUsec = GetCurCpuTimestamp();
    uint32_t strSize = strlen(traceStr);
    uint32_t posSize = 0U;
    TraceStr singleTraceStr;
    while (posSize < strSize) {
        uint32_t traceStrLen = std::min<uint32_t>(strSize - posSize, sizeof(singleTraceStr.transmit) - 1);
        CHK_SAFETY_FUNC_RET(
            memset_s(reinterpret_cast<void *>(&singleTraceStr.transmit), sizeof(singleTraceStr.transmit), 0,
            sizeof(singleTraceStr.transmit)));
        error_t ret = memcpy_s(singleTraceStr.transmit, traceStrLen, traceStr + posSize, traceStrLen);
        if (ret != EOK) {
            HCCL_ERROR("traceStr memcpy_s fail ret:%d", ret);
            return HCCL_E_MEMORY;
        }
        posSize += traceStrLen;
        auto traceStrRet =
            g_utraceSubmit(g_traceStrHandle, reinterpret_cast<const void *>(&singleTraceStr), sizeof(singleTraceStr));
        if (traceStrRet != TRACE_SUCCESS) {
            HCCL_ERROR("submittraceStr failed, ret:%d", traceStrRet);
            return HCCL_E_INTERNAL;
        }
        HCCL_INFO("Submit string: %s", singleTraceStr.transmit);
    }
    auto ctx = AicpuGetComContext();
    if (MC2AicpuUtils::IsDebugModeEquals(*ctx, MC2_DEBUG_TIME_TAKEN)) {
        ctx->acprof[g_proxLoopCnt].traceSubmitTime += GetCurCpuTimestamp() - startUsec;
    }
    return HCCL_SUCCESS;
}

HcclResult MC2TraceUtils::SubmitBatchSqeInfo()
{
    if (g_soHandle == nullptr) {
        return HCCL_SUCCESS;
    }
    uint64_t startUsec = GetCurCpuTimestamp();
    SqeContext *context = GetSqeContext();
    CHK_PTR_NULL(context->buffPtr);
    for (uint32_t streamId = 0U; streamId < AC_MAX_RANK_NUM; streamId++) {
        auto &buff = context->buffPtr[streamId];
        HCCL_INFO("sqTail: %u, sqHead: %u,  sqeCnt: %u, tailSqeTaskId: %u, tailSqeIdx: %u",
            buff.sqTail, buff.sqHead, buff.sqeCnt, buff.tailSqeTaskId, buff.tailSqeIdx);
        auto submitNum = std::min<uint32_t>(buff.tailSqeIdx - buff.sqeCnt, MAX_SQE_SUBMIT_NUM);
        for (uint32_t i = 0U; i <= submitNum / MAX_SQE_BATCH_SIZE; i++) {
            SqeBatchInfo sqeBatchInfo;
            for (uint32_t j = 0U; j < MAX_SQE_BATCH_SIZE; j++) {
                const auto idx = i * MAX_SQE_BATCH_SIZE + j;
                if (idx >= submitNum) {
                    break;
                }
                if (SqeContextUtils::QuerySqeInfoByTaskId(streamId, buff.tailSqeTaskId - submitNum + idx,
                    &sqeBatchInfo.sqeInfos[j]) != HCCL_SUCCESS) {
                    HCCL_WARNING("Query sqe info failed, stream:%u, id:%u", streamId, idx);
                    break;
                }
            }
            CHK_RET(Submit<SqeBatchInfo>(&sqeBatchInfo));
        }
    }
    auto ctx = AicpuGetComContext();
    if (MC2AicpuUtils::IsDebugModeEquals(*ctx, MC2_DEBUG_TIME_TAKEN)) {
        ctx->acprof[g_proxLoopCnt].traceSqeTime += GetCurCpuTimestamp() - startUsec;
    }
    return HCCL_SUCCESS;
}

HcclResult MC2TraceUtils::Save()
{
    if (g_soHandle == nullptr) {
        return HCCL_SUCCESS;
    }
    TraStatus ret = g_utraceEventReport(g_eventHandle);
    if (ret != TRACE_SUCCESS) {
        HCCL_ERROR("TraceEventReport failed, ret:%d", ret);
        return HCCL_E_INTERNAL;
    }
    return HCCL_SUCCESS;
}

HcclResult MC2TraceUtils::DestoryHandles()
{
    if (g_soHandle == nullptr) {
        return HCCL_SUCCESS;
    }
    g_utraceHandleDestroy(g_traceStrHandle);
    g_utraceHandleDestroy(g_traceTaskAndTilingDataHandle);
    g_utraceHandleDestroy(g_traceAicpuComDataHandle);
    g_utraceHandleDestroy(g_traceMsgInfoHandle);
    g_utraceHandleDestroy(g_traceSqeBatchInfoHandle);
    g_utraceEventDestroy(g_eventHandle);
    return HCCL_SUCCESS;
}

__attribute__ ((destructor)) void Finalize()
{
    if (g_soHandle != nullptr) {
        (void)mmDlclose(g_soHandle);
    }
    g_soHandle = nullptr;
}

// 显式实例化
template HcclResult MC2TraceUtils::Submit(const SqeBatchInfo * const);